from urllib.parse import urlencode
import requests
import json
from pyquery import PyQuery as pq
from pymongo import MongoClient

client = MongoClient()
db = client["weibo"]
collection = db["weibo"]

# 手机微博触屏版
base_url = "https://m.weibo.cn/api/container/getIndex?"

headers = {
    "MWeibo-Pwa":"1",
    "Referer":"https://m.weibo.cn/p/2304131650111241_-_WEIBO_SECOND_PROFILE_WEIBO",
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36",
    "X-Requested-With":"XMLHttpRequest",
    "X-XSRF-TOKEN":"a8050d",
}

def get_page(page):
    params = {
        "containerid": "2304131650111241_-_WEIBO_SECOND_PROFILE_WEIBO",
        "page_type": "03",
        "page": page,
    }
    url = base_url + urlencode(params)
    try:
        response = requests.get(url,headers=headers)
        if response.status_code == 200:
            return response.content.decode()
    except requests.ConnectionError as e:
        print('Error',e.args)

def parse_page(json_str):
    dict_ret = json.loads(json_str)
    # 遍历cards，然后获取mblog中的各个信息
    item_list = dict_ret.get("data").get("cards")
    for item in item_list:
        item = item.get("mblog")
        if item is not None:
            weibo = {}
            weibo["id"] = item.get("id")
            # 借助pyquery将正文中的HTML标签去掉
            weibo["text"] = pq(item.get("text")).text()
            weibo["点赞数"] = item.get("attitudes_count")
            weibo["评论数"] = item.get("comments_count")
            weibo["转发数"] = item.get("reposts_count")
            yield weibo

def save_to_mongo(results):
    for result in results:
        # if collection.insert(result):
        collection.insert(result)
        print("Saved to MongoDB")


def run(): #实现主要逻辑
    # 1.构造url
    # 2.发送请求，获取响应
    for page in range(1,11):
        json_str = get_page(page)
        # 3.提取数据
        results = parse_page(json_str)
    # 4.保存数据，点击下一页.
        save_to_mongo(results)

if __name__ == '__main__':
    run()

'''
注意1;使用urlencode()方法将参数转化为URL的GET请求参数，即类似于
     https://m.weibo.cn/api/container/getIndex?containerid=2304131650111241_-_WEIBO_SECOND_PROFILE_WEIBO&page_type=03&page=page
注意2：headers字典中键值对的值前面不能有空格（好像是用正则匹配替换之后会出现空格，不清楚原因）    
'''